/**
 * 
 */
package com.crawler;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;

/**
 * @author baohualiu
 * 
 */
public class Crawler {

	private final class Runner implements Runnable {

		@Override
		public void run() {

			while (true) {

				try {

					CrawlingURL curl = getFrontier().next();
					if (curl == null) {
						Thread.sleep(1000);
						continue;
					}

					if (getSeener().alreadySeen(curl))
						continue;

					FetchResult fr = getFetcher().fetch(curl);
					if (fr.lastResultOK()) {
						getPipeline().process(fr.getCrawlingDocument());
						
						getSeener().addSeen(curl);
					}

					Thread.sleep(1);

				} catch (InterruptedException e) {
					break;
				}

			} // while

		} // run

	}


	private int nThreads = 5;
	private Pipeline pipeline;
	private Fetcher fetcher;
	private Frontier frontier;
	private Seener seener;
	private List<CrawlerModule> modules = new ArrayList<>();

	/**
	 * 
	 */
	public Crawler() {
		this.frontier = new Frontier(this);
		this.fetcher = new Fetcher(this);
		this.pipeline = new Pipeline(this);
		this.seener = new Seener(this);
	}

	public void addModule(CrawlerModule m) {
		
		assert m != null;
		modules.add(m);
	}

	public void execute() throws InterruptedException {

		Executor ec = Executors.newFixedThreadPool(nThreads);

		for (int i = 0; i < nThreads; i++)
			ec.execute(new Runner());
	}

	public Fetcher getFetcher() {

		return fetcher;
	}

	public Frontier getFrontier() {

		return frontier;
	}

	public Pipeline getPipeline() {

		return pipeline;
	}

	private Seener getSeener() {

		return seener;
	}

}
